#####################
#Attitude data: IFLS#
#####################

oldwd = getwd()
#Directory
setwd('./data')

#Get objects
old_ws = ls()

#Load data
ifls = fread('./surveys/ifls_individual_data_07_14.csv')
ifls_l = fread('./surveys/ifls_commleader_data_07_14.csv')

ifls_2004 = ifls[year ==2007]
ifls_2009 = ifls[year == 2014]

ifls_l_2004 = ifls_l[year ==2007]
ifls_l_2009 = ifls_l[year == 2014]


############################
#Prepare to merge in dapils#
############################

#Load election-year kecamatan crosswalks
########################################
crosswalk_2004 = fread('./crosswalks/kecamatan_ifls_to_dprd2004.csv')
crosswalk_2009 = fread('./crosswalks/kecamatan_ifls_to_dprd2009.csv')
setnames(crosswalk_2004, c('target_kecamatan', 'from_kecamatan'), c('kecamatan_2004', 'kec_code'))
setnames(crosswalk_2009, c('target_kecamatan', 'from_kecamatan'), c('kecamatan_2009', 'kec_code'))

#Individual surveys
setkey(ifls_2004, kec_code)
setkey(ifls_2009, kec_code)
setkey(crosswalk_2004, kec_code)
setkey(crosswalk_2009, kec_code)

ifls_2004 = crosswalk_2004[ifls_2004]
ifls_2009 = crosswalk_2009[ifls_2009]

#Community Leader surveys
setkey(ifls_l_2004, kec_code)
setkey(ifls_l_2009, kec_code)
setkey(crosswalk_2004, kec_code)
setkey(crosswalk_2009, kec_code)

ifls_l_2004 = crosswalk_2004[ifls_l_2004]
ifls_l_2009 = crosswalk_2009[ifls_l_2009]

#Load kecamatan to DPRD crosswalks
##################################

#2004
kec_to_dprd_2004 = fread("./crosswalks/kecamatan_to_dprd2_2004.csv")
setnames(kec_to_dprd_2004, 'DAPIL.NUMBER', "DAPIL_NUMBER")
kec_to_dprd_2004[KECA %in% 1, KECA := 10]
kec_to_dprd_2004[, id_kec := paste0(PROP, sprintf("%02.f", as.numeric(KABU)), sprintf("%03.f", as.numeric(KECA)))]
kec_to_dprd_2004[, dapil := paste(KAB_NAME, DAPIL_NUMBER)]
kec_to_dprd_2004 = kec_to_dprd_2004[!is.na(DAPIL_NUMBER)]
kec_to_dprd_2004 = kec_to_dprd_2004[, 
                                    list(provinsi = PROV_NAME,
                                         id_prov = PROP,
                                         kabupaten = KAB_NAME,
                                         id_kab = KAB_CODE,
                                         id_kec = as.numeric(id_kec), dapil)]
kec_to_dprd_2004[provinsi %in% "IRIAN JAYA BARAT", id_prov := 91]


#2009
kec_to_dprd_2009 = fread("./crosswalks/kecamatan_to_dprd2_2009.csv")
kec_to_dprd_2009 = kec_to_dprd_2009[, 
                                    list(provinsi = provinsi, id_prov = provno,
                                         kabupaten = kabkot, id_kab = id_kab,
                                         id_kec = id_kec, dapil = label)]



#Merge in dapil codes
#####################

#Individual Surveys
setkey(ifls_2004, kecamatan_2004)
setkey(kec_to_dprd_2004, id_kec)

setkey(ifls_2009, kecamatan_2009)
setkey(kec_to_dprd_2009, id_kec)

ifls_2004 = kec_to_dprd_2004[ifls_2004]
ifls_2009 = kec_to_dprd_2009[ifls_2009]


#Community Leader Surveys
setkey(ifls_l_2004, kecamatan_2004)
setkey(kec_to_dprd_2004, id_kec)

setkey(ifls_l_2009, kecamatan_2009)
setkey(kec_to_dprd_2009, id_kec)

ifls_l_2004 = kec_to_dprd_2004[ifls_l_2004]
ifls_l_2009 = kec_to_dprd_2009[ifls_l_2009]


####################
#Individual Surveys#
####################

#Select data
ifls_2004 = ifls_2004[!is.na(id_kab), list(provinsi, id_prov , kabupaten, id_kab, dapil,  
                                                     relg = tr12, 
                                                     trust_relg_more = tr23,
                                                     live_in_village = tr24,
                                                     live_in_neighbor = tr25,
                                                     rents_room = tr26,
                                                     marries_child = tr27,
                                                     build_relg_estab = tr28,
                                                     relg_candidate = tr29,
                                                     trust_ethnicity_more = tr03,
                                                     village_safe = tr06,
                                                     noone_present = str_detect(cp1, "A"),
                                                     spouse_present = str_detect(cp1, "D"),
                                                     adult_hh_present = str_detect(cp1, "E"),
                                                     adult_nh_present = str_detect(cp1, "F"),                                                     pidlink,
                                                     election_cycle = 2004)]


ifls_2009 = ifls_2009[!is.na(id_kab), list(provinsi, id_prov , kabupaten, id_kab, dapil,  
                                           relg = tr12, 
                                           trust_relg_more = tr23,
                                           live_in_village = tr24,
                                           live_in_neighbor = tr25,
                                           rents_room = tr26,
                                           marries_child = tr27,
                                           build_relg_estab = tr28,
                                           relg_candidate = tr29,
                                           trust_ethnicity_more = tr03,
                                           village_safe = tr06,
                                           noone_present = str_detect(cp1, "A"),
                                           spouse_present = str_detect(cp1, "D"),
                                           adult_hh_present = str_detect(cp1, "E"),
                                           adult_nh_present = str_detect(cp1, "F"),
                                           pidlink,
                                           election_cycle = 2009)]

#Merge individual ifls
ifls_dapil = rbindlist(list(ifls_2004, ifls_2009), use.names = T)

#Clean up 
vars = c('relg', 'trust_relg_more', 
         'live_in_village', 'live_in_neighbor', 'rents_room', 'marries_child', 'build_relg_estab',
         'relg_candidate', 'trust_ethnicity_more', 'village_safe')
ifls_dapil[, c(vars) := lapply(.SD, function(x) str_extract(x, "^\\d+")), .SDcols = vars]
ifls_dapil[,c(vars) := lapply(.SD, as.numeric), .SDcols = vars]



#Clean intolerance data

intol_vars = c('live_in_village', 'live_in_neighbor', 'rents_room', 'marries_child', 'build_relg_estab')

#Switch direction for build_relg_estab in 2004
ifls_dapil[election_cycle %in% 2004, c(intol_vars[5]) := lapply(.SD, function(x) -1*(x-5)), .SDcols = intol_vars[5]]


##########################
#Community Leader Surveys#
##########################
ifls_l_2004 = ifls_l_2004[!is.na(id_kab), list(provinsi, id_prov, kabupaten, id_kab, dapil,
                                 village_safe = tr05,
                                 civil_strife = (tr13 == 1),
                                 trust_ethnicity_more = tr17,
                                 trust_religion_more = tr18,
                                 election_cycle = 2004)]

ifls_l_2009 = ifls_l_2009[!is.na(id_kab), list(provinsi, id_prov, kabupaten, id_kab, dapil,
                                 village_safe = tr05,
                                 civil_strife = (tr13 == 1),
                                 trust_ethnicity_more = tr17,
                                 trust_religion_more = tr18,
                                 election_cycle = 2009)]

ifls_l_dapil = rbindlist(list(ifls_l_2004, ifls_l_2009), use.names = T)

#########################
#Merge in dapil clusters#
#########################
dapil_clusters = fread('./crosswalks/dapil_clusters.csv')

setkey(dapil_clusters, election_cycle, id_kab, dapil)
setkey(ifls_dapil, election_cycle, id_kab, dapil)

ifls_dapil = dapil_clusters[ifls_dapil]
ifls_dapil[is.na(cluster), cluster := (-1*(.GRP)) %>% as.integer, by = list(id_kab, dapil, election_cycle)]


setkey(dapil_clusters, election_cycle, id_kab, dapil)
setkey(ifls_l_dapil, election_cycle, id_kab, dapil)

ifls_l_dapil = dapil_clusters[ifls_l_dapil]
ifls_l_dapil[is.na(cluster), cluster := (-1*(.GRP)) %>% as.integer, by = list(id_kab, dapil, election_cycle)]




drop = setdiff(ls(), c(old_ws, 'ifls_dapil', 'ifls_l_dapil')) 
rm(list = drop)
setwd(oldwd)

